import os, sys, time, asyncio, aiohttp

import re

import pymysql
import pypinyin
import tqdm

#           http://pc6.com/down.asp?id=400328
#           http://pc6.com/down.asp?id=400328
#           http://www.pc6.com/azyx/55037.html
BASE_URL = 'http://www.pc6.com/azyx/'  # 下载url
DEST_DIR = 'downloads/'  # 保存目录
headers = {
    r'Host': 'www.pc6.com',
    r'Connection': 'keep-alive',
    r'Cache-Control': 'max-age=0',
    r'Upgrade-Insecure-Requests': '1',
    r'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
    r'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    r'Accept-Encoding': 'gzip, deflate, sdch',
    r'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
    r'Cookie': 'UM_distinctid=165133fd601337-0c16e337836a12-454c092b-100200-165133fd602440; Hm_lvt_1d9e6afba99e74b0adff97c2dab13432=1533631294; Hm_lvt_0de7b5f7e443add55367defe1b3d292c=1533631995; Hm_lvt_e89c3a7bcca9d2b77d9d7073cb36fa0a=1533634175; ASPSESSIONIDCARACTCS=ANIKOIKANJAJMOINMBAEDGJG; Hm_lvt_2337f2e83d566c915b997eefb8fad98e=1531206571,1533626266,1533699370; CNZZDATA30064912=cnzz_eid%3D1499925981-1533626073-%26ntime%3D1533704518; Hm_lvt_027f0b1690460ddcb51c3e39eff0243e=1533626288,1533706893; Hm_lpvt_027f0b1690460ddcb51c3e39eff0243e=1533706893; bdshare_firstime=1533706907886; Hm_lvt_5bec470cb053e1006cd204355bdf43b5=1533706941; Hm_lvt_7c9a22609a0017275f143d3b1af47740=1533706951; __tins__19095269=%7B%22sid%22%3A%201533707381526%2C%20%22vd%22%3A%201%2C%20%22expires%22%3A%201533709181526%7D; __51cke__=; __51laig__=1; Hm_lvt_31c9d092c6c1f89069c4ddbddbb4e61c=1533707410; Hm_lpvt_31c9d092c6c1f89069c4ddbddbb4e61c=1533707410; CNZZDATA30069696=cnzz_eid%3D1938715584-1533703199-null%26ntime%3D1533703199; Hm_lvt_37b5bba0728480a528bdca94eeccfefb=1533707416; Hm_lpvt_37b5bba0728480a528bdca94eeccfefb=1533707416; Hm_lpvt_7c9a22609a0017275f143d3b1af47740=1533708290; CNZZDATA30098597=cnzz_eid%3D714248472-1533626953-%26ntime%3D1533703348; Hm_lvt_03080c51591948a3667414db7a193188=1533628767,1533708432; Hm_lpvt_03080c51591948a3667414db7a193188=1533708453; Hm_lpvt_5bec470cb053e1006cd204355bdf43b5=1533709621; Hm_lpvt_2337f2e83d566c915b997eefb8fad98e=1533709621',
}


# 获取链接,下载文件
async def fetch(session: aiohttp.ClientSession, url: str, path: str, flag: str):
    # print(flag, ' 开始下载')
    # url = "http://www.pc6.com/azyx/401456.html"
    async with session.get(url, headers=headers) as resp:
        try:
            html = await resp.read()
            html = html.decode('gb2312', 'ignore')
            typelocation = re.search(r'<a href="/default.html">首页</a>.*?安卓手游</a>', html)
            if typelocation != None:
                apkSizeList = re.findall('版本.*大小：.*?(\d*\.?\d*)M', html)
                if apkSizeList.__len__() > 0:
                    if (float(apkSizeList[0])) < 100.0:
                        apkUrlList = re.findall('dd id="dinfo" sid="(.*?)" cid', html)
                        appName = re.findall('<p id="fkey">(.*?)</p>', html)
                        # path = "E:/app/apk/%s.apk" % (appName[0] if appName.__len__() > 0 else str(0))
                        if apkUrlList.__len__() > 0:
                            print('正在下载>>%s>>http://pc6.com/down.asp?id=%s' % (appName, apkUrlList[0]))
                            installNO(cursor, db, flag, (appName[0] if appName.__len__() > 0 else str(0)),
                                      apkUrlList[0])
                        else:
                            print("%s未找到下载地址" % flag)
                    else:
                        print("%s超过指定大小" % flag)
                else:
                    print("%s未找到大小信息" % flag)
            else:
                print("%s不是游戏" % flag)
        except Exception as e:
            aiohttp.traceback.print_exc()
            print("boom!!!%s" % url)
    return flag


async def begin_download(sem, session: aiohttp.ClientSession, url: str, path: str, flag: str):
    # 控制协程并发数量
    with (await sem):
        return await fetch(session, url, path, flag)


async def download(sem: asyncio.Semaphore):
    tasks = []
    async with aiohttp.ClientSession() as session:
        for flag in FLAGS:
            # 创建路径以及url
            path = os.path.join(DEST_DIR, '%s.html' % flag)
            url = '{}{cc}.html'.format(BASE_URL, cc=flag)
            # 构造一个协程列表
            tasks.append(asyncio.ensure_future(begin_download(sem, session, url, path, flag)))
        # 等待返回结果
        tasks_iter = asyncio.as_completed(tasks)
        # 创建一个进度条
        fk_task_iter = tqdm.tqdm(tasks_iter, total=len(FLAGS))
        count = 0
        for coroutine in fk_task_iter:
            res = await coroutine
            if count > 2000:
                count = 0
                time.sleep(5)
                print("休息5秒")
            # print(res, '下载完成')
            count += 1


def hp(word):
    s = ''
    for i in pypinyin.pinyin(word, style=pypinyin.NORMAL):
        s += ''.join(i)
    return s


# 查询上次最后一次的序号
def selectNo(cursor):
    cursor.execute("select *  from pc6 where _id=(select max(_id)  from apk3)")
    maxNO = cursor.fetchall()
    if maxNO.__len__() > 0:
        firstNo = maxNO[0][1]
        return int(firstNo) + 1
    return 0


# 插入序号
def installNO(cursor, db, NO, appName="", downAddress=""):
    sql = "insert into pc6(No,appName,appNamePy,downAddress) values (%d,'%s','%s','%s')" % (NO, appName, hp(appName), downAddress)
    cursor.execute(sql)
    db.commit()
    db.rollback()


if __name__ == '__main__':
    db = pymysql.connect("localhost", "root", "164531", "appmsg")
    cursor = db.cursor()
    firstNo = selectNo(cursor)
    FLAGS = []
    # 0~5000
    for i in range(300000, 400000):
        FLAGS.append(i)
    # 创建目录
    os.makedirs(DEST_DIR, exist_ok=True)
    # 获取事件循环
    lp = asyncio.get_event_loop()
    start = time.time()
    # 创建一个信号量以防止DDos
    sem = asyncio.Semaphore(4)
    lp.run_until_complete(download(sem))
    end = time.time()
    lp.close()
    print('耗时:', end - start)
